

<!DOCTYPE html>
<html class="writer-html5" lang="en" >
<head>
  <meta charset="utf-8" />
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  
  <title>Troubleshooting &mdash; Ceph Documentation</title>
  

  
  <link rel="stylesheet" href="../../_static/ceph.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/graphviz.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/custom.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../../_static/favicon.ico"/>
  

  
  

  

  
  <!--[if lt IE 9]>
    <script src="../../_static/js/html5shiv.min.js"></script>
  <![endif]-->
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
        <script src="../../_static/jquery.js"></script>
        <script src="../../_static/underscore.js"></script>
        <script src="../../_static/doctools.js"></script>
    
    <script type="text/javascript" src="../../_static/js/theme.js"></script>

    
    <link rel="index" title="Index" href="../../genindex/" />
    <link rel="search" title="Search" href="../../search/" />
    <link rel="next" title="CEPHADM 开发者文档" href="../../dev/cephadm/" />
    <link rel="prev" title="Basic Ceph Client Setup" href="../client-setup/" /> 
</head>

<body class="wy-body-for-nav">

   
  <header class="top-bar">
    

















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../../" class="icon icon-home"></a> &raquo;</li>
        
          <li><a href="../">Cephadm</a> &raquo;</li>
        
      <li>Troubleshooting</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
          
            <a href="../../_sources/cephadm/troubleshooting.rst.txt" rel="nofollow"> View page source</a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
  </header>
  <div class="wy-grid-for-nav">
    
    <nav data-toggle="wy-nav-shift" class="wy-nav-side">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search"  style="background: #eee" >
          

          
            <a href="../../">
          

          
            
            <img src="../../_static/logo.png" class="logo" alt="Logo"/>
          
          </a>

          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../search/" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <ul class="current">
<li class="toctree-l1"><a class="reference internal" href="../../start/intro/">Ceph 简介</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../install/">安装 Ceph</a></li>
<li class="toctree-l1 current"><a class="reference internal" href="../">Cephadm</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="../compatibility/">Compatibility and Stability</a></li>
<li class="toctree-l2"><a class="reference internal" href="../install/">部署个全新的 Ceph 集群</a></li>
<li class="toctree-l2"><a class="reference internal" href="../adoption/">现有集群切换到 cephadm</a></li>
<li class="toctree-l2"><a class="reference internal" href="../host-management/">Host Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../services/">Service Management</a></li>
<li class="toctree-l2"><a class="reference internal" href="../upgrade/">Ceph 的升级</a></li>
<li class="toctree-l2"><a class="reference internal" href="../operations/">Cephadm operations</a></li>
<li class="toctree-l2"><a class="reference internal" href="../client-setup/">Client Setup</a></li>
<li class="toctree-l2 current"><a class="current reference internal" href="#">Troubleshooting</a><ul>
<li class="toctree-l3"><a class="reference internal" href="#pausing-or-disabling-cephadm">Pausing or disabling cephadm</a></li>
<li class="toctree-l3"><a class="reference internal" href="#per-service-and-per-daemon-events">Per-service and per-daemon events</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#listing-service-events">Listing service events</a></li>
<li class="toctree-l4"><a class="reference internal" href="#listing-daemon-events">Listing daemon events</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#checking-cephadm-logs">Checking cephadm logs</a></li>
<li class="toctree-l3"><a class="reference internal" href="#gathering-log-files">Gathering log files</a></li>
<li class="toctree-l3"><a class="reference internal" href="#collecting-systemd-status">Collecting systemd status</a></li>
<li class="toctree-l3"><a class="reference internal" href="#list-all-downloaded-container-images">List all downloaded container images</a></li>
<li class="toctree-l3"><a class="reference internal" href="#manually-running-containers">Manually running containers</a></li>
<li class="toctree-l3"><a class="reference internal" href="#ssh-errors">ssh errors</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#verifying-that-the-public-key-is-listed-in-the-authorized-keys-file">Verifying that the Public Key is Listed in the authorized_keys file</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#failed-to-infer-cidr-network-error">Failed to infer CIDR network error</a></li>
<li class="toctree-l3"><a class="reference internal" href="#accessing-the-admin-socket">Accessing the admin socket</a></li>
<li class="toctree-l3"><a class="reference internal" href="#calling-miscellaneous-ceph-tools">Calling miscellaneous ceph tools</a></li>
<li class="toctree-l3"><a class="reference internal" href="#restoring-the-mon-quorum">Restoring the MON quorum</a></li>
<li class="toctree-l3"><a class="reference internal" href="#manually-deploying-a-mgr-daemon">Manually deploying a MGR daemon</a></li>
<li class="toctree-l3"><a class="reference internal" href="#analyzing-core-dumps">Analyzing core dumps</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="../../dev/cephadm/">Cephadm Feature Planning</a></li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="../../rados/">Ceph 存储集群</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cephfs/">Ceph 文件系统</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../rbd/">Ceph 块设备</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../radosgw/">Ceph 对象网关</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mgr/">Ceph 管理器守护进程</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mgr/dashboard/">Ceph 仪表盘</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../api/">API 文档</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../architecture/">体系结构</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/developer_guide/">开发者指南</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dev/internals/">Ceph 内幕</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../governance/">项目管理</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../foundation/">Ceph 基金会</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ceph-volume/">ceph-volume</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../releases/general/">Ceph 版本（总目录）</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../releases/">Ceph 版本（索引）</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../security/">Security</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../glossary/">Ceph 术语</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jaegertracing/">Tracing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../translation_cn/">中文版翻译资源</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">

      
      <nav class="wy-nav-top" aria-label="top navigation">
        
          <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
          <a href="../../">Ceph</a>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
<div id="dev-warning" class="admonition note">
  <p class="first admonition-title">Notice</p>
  <p class="last">This document is for a development version of Ceph.</p>
</div>
  <div id="docubetter" align="right" style="padding: 5px; font-weight: bold;">
    <a href="https://pad.ceph.com/p/Report_Documentation_Bugs">Report a Documentation Bug</a>
  </div>

  
  <div class="section" id="troubleshooting">
<h1>Troubleshooting<a class="headerlink" href="#troubleshooting" title="Permalink to this headline">¶</a></h1>
<p>You might need to investigate why a cephadm command failed
or why a certain service no longer runs properly.</p>
<p>Cephadm deploys daemons as containers. This means that
troubleshooting those containerized daemons might work
differently than you expect (and that is certainly true if
you expect this troubleshooting to work the way that
troubleshooting does when the daemons involved aren’t
containerized).</p>
<p>Here are some tools and commands to help you troubleshoot
your Ceph environment.</p>
<div class="section" id="pausing-or-disabling-cephadm">
<span id="cephadm-pause"></span><h2>Pausing or disabling cephadm<a class="headerlink" href="#pausing-or-disabling-cephadm" title="Permalink to this headline">¶</a></h2>
<p>If something goes wrong and cephadm is behaving badly, you can
pause most of the Ceph cluster’s background activity by running
the following command:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><style type="text/css">
span.prompt1:before {
  content: "# ";
}
</style><span class="prompt1">ceph orch pause</span>
</pre></div></div><p>This stops all changes in the Ceph cluster, but cephadm will
still periodically check hosts to refresh its inventory of
daemons and devices.  You can disable cephadm completely by
running the following commands:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">ceph orch <span class="nb">set</span> backend <span class="s1">&#39;&#39;</span></span>
<span class="prompt1">ceph mgr module disable cephadm</span>
</pre></div></div><p>These commands disable all of the <code class="docutils literal notranslate"><span class="pre">ceph</span> <span class="pre">orch</span> <span class="pre">...</span></code> CLI commands.
All previously deployed daemon containers continue to exist and
will start as they did before you ran these commands.</p>
<p>See <a class="reference internal" href="../services/#cephadm-spec-unmanaged"><span class="std std-ref">Disabling automatic deployment of daemons</span></a> for information on disabling
individual services.</p>
</div>
<div class="section" id="per-service-and-per-daemon-events">
<h2>Per-service and per-daemon events<a class="headerlink" href="#per-service-and-per-daemon-events" title="Permalink to this headline">¶</a></h2>
<p>In order to help with the process of debugging failed daemon
deployments, cephadm stores events per service and per daemon.
These events often contain information relevant to
troubleshooting
your Ceph cluster.</p>
<div class="section" id="listing-service-events">
<h3>Listing service events<a class="headerlink" href="#listing-service-events" title="Permalink to this headline">¶</a></h3>
<p>To see the events associated with a certain service, run a
command of the and following form:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">ceph orch ls --service_name<span class="o">=</span>&lt;service-name&gt; --format yaml</span>
</pre></div></div><p>This will return something in the following form:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">service_type</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">alertmanager</span>
<span class="nt">service_name</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">alertmanager</span>
<span class="nt">placement</span><span class="p">:</span>
  <span class="nt">hosts</span><span class="p">:</span>
  <span class="p p-Indicator">-</span> <span class="l l-Scalar l-Scalar-Plain">unknown_host</span>
<span class="nt">status</span><span class="p">:</span>
  <span class="l l-Scalar l-Scalar-Plain">...</span>
  <span class="l l-Scalar l-Scalar-Plain">running</span><span class="p p-Indicator">:</span> <span class="l l-Scalar l-Scalar-Plain">1</span>
  <span class="nt">size</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">1</span>
<span class="nt">events</span><span class="p">:</span>
<span class="p p-Indicator">-</span> <span class="l l-Scalar l-Scalar-Plain">2021-02-01T08:58:02.741162 service:alertmanager [INFO] &quot;service was created&quot;</span>
<span class="p p-Indicator">-</span> <span class="s">&#39;2021-02-01T12:09:25.264584</span><span class="nv"> </span><span class="s">service:alertmanager</span><span class="nv"> </span><span class="s">[ERROR]</span><span class="nv"> </span><span class="s">&quot;Failed</span><span class="nv"> </span><span class="s">to</span><span class="nv"> </span><span class="s">apply:</span><span class="nv"> </span><span class="s">Cannot</span>
  <span class="s">place</span><span class="nv"> </span><span class="s">&lt;AlertManagerSpec</span><span class="nv"> </span><span class="s">for</span><span class="nv"> </span><span class="s">service_name=alertmanager&gt;</span><span class="nv"> </span><span class="s">on</span><span class="nv"> </span><span class="s">unknown_host:</span><span class="nv"> </span><span class="s">Unknown</span><span class="nv"> </span><span class="s">hosts&quot;&#39;</span>
</pre></div>
</div>
</div>
<div class="section" id="listing-daemon-events">
<h3>Listing daemon events<a class="headerlink" href="#listing-daemon-events" title="Permalink to this headline">¶</a></h3>
<p>To see the events associated with a certain daemon, run a
command of the and following form:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1">ceph orch ps --service-name &lt;service-name&gt; --daemon-id &lt;daemon-id&gt; --format yaml</span>
</pre></div></div><p>This will return something in the following form:</p>
<div class="highlight-yaml notranslate"><div class="highlight"><pre><span></span><span class="nt">daemon_type</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">mds</span>
<span class="nt">daemon_id</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">cephfs.hostname.ppdhsz</span>
<span class="nt">hostname</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">hostname</span>
<span class="nt">status_desc</span><span class="p">:</span> <span class="l l-Scalar l-Scalar-Plain">running</span>
<span class="nn">...</span>
<span class="nt">events</span><span class="p">:</span>
<span class="p p-Indicator">-</span> <span class="l l-Scalar l-Scalar-Plain">2021-02-01T08:59:43.845866 daemon:mds.cephfs.hostname.ppdhsz [INFO] &quot;Reconfigured</span>
  <span class="l l-Scalar l-Scalar-Plain">mds.cephfs.hostname.ppdhsz on host &#39;hostname&#39;&quot;</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="checking-cephadm-logs">
<h2>Checking cephadm logs<a class="headerlink" href="#checking-cephadm-logs" title="Permalink to this headline">¶</a></h2>
<p>To learn how to monitor the cephadm logs as they are generated, read <a class="reference internal" href="../operations/#watching-cephadm-logs"><span class="std std-ref">Watching cephadm log messages</span></a>.</p>
<p>If your Ceph cluster has been configured to log events to files, there will exist a
cephadm log file called <code class="docutils literal notranslate"><span class="pre">ceph.cephadm.log</span></code> on all monitor hosts (see
<a class="reference internal" href="../operations/#cephadm-logs"><span class="std std-ref">Ceph daemon logs</span></a> for a more complete explanation of this).</p>
</div>
<div class="section" id="gathering-log-files">
<h2>Gathering log files<a class="headerlink" href="#gathering-log-files" title="Permalink to this headline">¶</a></h2>
<p>Use journalctl to gather the log files of all daemons:</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>By default cephadm now stores logs in journald. This means
that you will no longer find daemon logs in <code class="docutils literal notranslate"><span class="pre">/var/log/ceph/</span></code>.</p>
</div>
<p>To read the log file of one specific daemon, run:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cephadm</span> <span class="n">logs</span> <span class="o">--</span><span class="n">name</span> <span class="o">&lt;</span><span class="n">name</span><span class="o">-</span><span class="n">of</span><span class="o">-</span><span class="n">daemon</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>Note: this only works when run on the same host where the daemon is running. To
get logs of a daemon running on a different host, give the <code class="docutils literal notranslate"><span class="pre">--fsid</span></code> option:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cephadm</span> <span class="n">logs</span> <span class="o">--</span><span class="n">fsid</span> <span class="o">&lt;</span><span class="n">fsid</span><span class="o">&gt;</span> <span class="o">--</span><span class="n">name</span> <span class="o">&lt;</span><span class="n">name</span><span class="o">-</span><span class="n">of</span><span class="o">-</span><span class="n">daemon</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>where the <code class="docutils literal notranslate"><span class="pre">&lt;fsid&gt;</span></code> corresponds to the cluster ID printed by <code class="docutils literal notranslate"><span class="pre">ceph</span> <span class="pre">status</span></code>.</p>
<p>To fetch all log files of all daemons on a given host, run:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>for name in $(cephadm ls | jq -r &#39;.[].name&#39;) ; do
  cephadm logs --fsid &lt;fsid&gt; --name &quot;$name&quot; &gt; $name;
done
</pre></div>
</div>
</div>
<div class="section" id="collecting-systemd-status">
<h2>Collecting systemd status<a class="headerlink" href="#collecting-systemd-status" title="Permalink to this headline">¶</a></h2>
<p>To print the state of a systemd unit, run:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">systemctl</span> <span class="n">status</span> <span class="s2">&quot;ceph-$(cephadm shell ceph fsid)@&lt;service name&gt;.service&quot;</span><span class="p">;</span>
</pre></div>
</div>
<p>To fetch all state of all daemons of a given host, run:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>fsid=&quot;$(cephadm shell ceph fsid)&quot;
for name in $(cephadm ls | jq -r &#39;.[].name&#39;) ; do
  systemctl status &quot;ceph-$fsid@$name.service&quot; &gt; $name;
done
</pre></div>
</div>
</div>
<div class="section" id="list-all-downloaded-container-images">
<h2>List all downloaded container images<a class="headerlink" href="#list-all-downloaded-container-images" title="Permalink to this headline">¶</a></h2>
<p>To list all container images that are downloaded on a host:</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><code class="docutils literal notranslate"><span class="pre">Image</span></code> might also be called <cite>ImageID</cite></p>
</div>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">podman</span> <span class="n">ps</span> <span class="o">-</span><span class="n">a</span> <span class="o">--</span><span class="nb">format</span> <span class="n">json</span> <span class="o">|</span> <span class="n">jq</span> <span class="s1">&#39;.[].Image&#39;</span>
<span class="s2">&quot;docker.io/library/centos:8&quot;</span>
<span class="s2">&quot;registry.opensuse.org/opensuse/leap:15.2&quot;</span>
</pre></div>
</div>
</div>
<div class="section" id="manually-running-containers">
<h2>Manually running containers<a class="headerlink" href="#manually-running-containers" title="Permalink to this headline">¶</a></h2>
<p>Cephadm writes small wrappers that run a containers. Refer to
<code class="docutils literal notranslate"><span class="pre">/var/lib/ceph/&lt;cluster-fsid&gt;/&lt;service-name&gt;/unit.run</span></code> for the
container execution command.</p>
</div>
<div class="section" id="ssh-errors">
<span id="cephadm-ssh-errors"></span><h2>ssh errors<a class="headerlink" href="#ssh-errors" title="Permalink to this headline">¶</a></h2>
<p>Error message:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">execnet</span><span class="o">.</span><span class="n">gateway_bootstrap</span><span class="o">.</span><span class="n">HostNotFound</span><span class="p">:</span> <span class="o">-</span><span class="n">F</span> <span class="o">/</span><span class="n">tmp</span><span class="o">/</span><span class="n">cephadm</span><span class="o">-</span><span class="n">conf</span><span class="o">-</span><span class="mi">73</span><span class="n">z09u6g</span> <span class="o">-</span><span class="n">i</span> <span class="o">/</span><span class="n">tmp</span><span class="o">/</span><span class="n">cephadm</span><span class="o">-</span><span class="n">identity</span><span class="o">-</span><span class="n">ky7ahp_5</span> <span class="n">root</span><span class="o">@</span><span class="mf">10.10.1.2</span>
<span class="o">...</span>
<span class="k">raise</span> <span class="n">OrchestratorError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span> <span class="kn">from</span> <span class="nn">e</span>
<span class="n">orchestrator</span><span class="o">.</span><span class="n">_interface</span><span class="o">.</span><span class="n">OrchestratorError</span><span class="p">:</span> <span class="n">Failed</span> <span class="n">to</span> <span class="n">connect</span> <span class="n">to</span> <span class="mf">10.10.1.2</span> <span class="p">(</span><span class="mf">10.10.1.2</span><span class="p">)</span><span class="o">.</span>
<span class="n">Please</span> <span class="n">make</span> <span class="n">sure</span> <span class="n">that</span> <span class="n">the</span> <span class="n">host</span> <span class="ow">is</span> <span class="n">reachable</span> <span class="ow">and</span> <span class="n">accepts</span> <span class="n">connections</span> <span class="n">using</span> <span class="n">the</span> <span class="n">cephadm</span> <span class="n">SSH</span> <span class="n">key</span>
<span class="o">...</span>
</pre></div>
</div>
<p>Things users can do:</p>
<ol class="arabic">
<li><p>Ensure cephadm has an SSH identity key:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span><span class="o">~</span><span class="p">]</span><span class="c1"># cephadm shell -- ceph config-key get mgr/cephadm/ssh_identity_key &gt; ~/cephadm_private_key</span>
<span class="n">INFO</span><span class="p">:</span><span class="n">cephadm</span><span class="p">:</span><span class="n">Inferring</span> <span class="n">fsid</span> <span class="n">f8edc08a</span><span class="o">-</span><span class="mi">7</span><span class="n">f17</span><span class="o">-</span><span class="mi">11</span><span class="n">ea</span><span class="o">-</span><span class="mi">8707</span><span class="o">-</span><span class="mi">000</span><span class="n">c2915dd98</span>
<span class="n">INFO</span><span class="p">:</span><span class="n">cephadm</span><span class="p">:</span><span class="n">Using</span> <span class="n">recent</span> <span class="n">ceph</span> <span class="n">image</span> <span class="n">docker</span><span class="o">.</span><span class="n">io</span><span class="o">/</span><span class="n">ceph</span><span class="o">/</span><span class="n">ceph</span><span class="p">:</span><span class="n">v15</span> <span class="n">obtained</span> <span class="s1">&#39;mgr/cephadm/ssh_identity_key&#39;</span>
<span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span> <span class="c1"># chmod 0600 ~/cephadm_private_key</span>
</pre></div>
</div>
</li>
</ol>
<blockquote>
<div><p>If this fails, cephadm doesn’t have a key. Fix this by running the following command:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># cephadm shell -- ceph cephadm generate-ssh-key</span>
</pre></div>
</div>
<p>or:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># cat ~/cephadm_private_key | cephadm shell -- ceph cephadm set-ssk-key -i -</span>
</pre></div>
</div>
</div></blockquote>
<ol class="arabic" start="2">
<li><p>Ensure that the ssh config is correct:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># cephadm shell -- ceph cephadm get-ssh-config &gt; config</span>
</pre></div>
</div>
</li>
<li><p>Verify that we can connect to the host:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># ssh -F config -i ~/cephadm_private_key root@mon1</span>
</pre></div>
</div>
</li>
</ol>
<div class="section" id="verifying-that-the-public-key-is-listed-in-the-authorized-keys-file">
<h3>Verifying that the Public Key is Listed in the authorized_keys file<a class="headerlink" href="#verifying-that-the-public-key-is-listed-in-the-authorized-keys-file" title="Permalink to this headline">¶</a></h3>
<p>To verify that the public key is in the authorized_keys file, run the following commands:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># cephadm shell -- ceph cephadm get-pub-key &gt; ~/ceph.pub</span>
<span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># grep &quot;`cat ~/ceph.pub`&quot;  /root/.ssh/authorized_keys</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="failed-to-infer-cidr-network-error">
<h2>Failed to infer CIDR network error<a class="headerlink" href="#failed-to-infer-cidr-network-error" title="Permalink to this headline">¶</a></h2>
<p>If you see this error:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ERROR</span><span class="p">:</span> <span class="n">Failed</span> <span class="n">to</span> <span class="n">infer</span> <span class="n">CIDR</span> <span class="n">network</span> <span class="k">for</span> <span class="n">mon</span> <span class="n">ip</span> <span class="o">***</span><span class="p">;</span> <span class="k">pass</span> <span class="o">--</span><span class="n">skip</span><span class="o">-</span><span class="n">mon</span><span class="o">-</span><span class="n">network</span> <span class="n">to</span> <span class="n">configure</span> <span class="n">it</span> <span class="n">later</span>
</pre></div>
</div>
<p>Or this error:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">Must</span> <span class="nb">set</span> <span class="n">public_network</span> <span class="n">config</span> <span class="n">option</span> <span class="ow">or</span> <span class="n">specify</span> <span class="n">a</span> <span class="n">CIDR</span> <span class="n">network</span><span class="p">,</span> <span class="n">ceph</span> <span class="n">addrvec</span><span class="p">,</span> <span class="ow">or</span> <span class="n">plain</span> <span class="n">IP</span>
</pre></div>
</div>
<p>This means that you must run a command of this form:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ceph</span> <span class="n">config</span> <span class="nb">set</span> <span class="n">mon</span> <span class="n">public_network</span> <span class="o">&lt;</span><span class="n">mon_network</span><span class="o">&gt;</span>
</pre></div>
</div>
<p>For more detail on operations of this kind, see <a class="reference internal" href="../services/mon/#deploy-additional-monitors"><span class="std std-ref">Deploying additional monitors</span></a></p>
</div>
<div class="section" id="accessing-the-admin-socket">
<h2>Accessing the admin socket<a class="headerlink" href="#accessing-the-admin-socket" title="Permalink to this headline">¶</a></h2>
<p>Each Ceph daemon provides an admin socket that bypasses the
MONs (See <a class="reference internal" href="../../rados/operations/monitoring/#rados-monitoring-using-admin-socket"><span class="std std-ref">使用管理套接字</span></a>).</p>
<p>To access the admin socket, first enter the daemon container on the host:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="p">[</span><span class="n">root</span><span class="nd">@mon1</span> <span class="o">~</span><span class="p">]</span><span class="c1"># cephadm enter --name &lt;daemon-name&gt;</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@mon1</span> <span class="o">/</span><span class="p">]</span><span class="c1"># ceph --admin-daemon /var/run/ceph/ceph-&lt;daemon-name&gt;.asok config show</span>
</pre></div>
</div>
</div>
<div class="section" id="calling-miscellaneous-ceph-tools">
<h2>Calling miscellaneous ceph tools<a class="headerlink" href="#calling-miscellaneous-ceph-tools" title="Permalink to this headline">¶</a></h2>
<p>To call miscellaneous like <code class="docutils literal notranslate"><span class="pre">ceph-objectstore-tool</span></code> or
<code class="docutils literal notranslate"><span class="pre">ceph-monstore-tool</span></code>, you can run them by calling
<code class="docutils literal notranslate"><span class="pre">cephadm</span> <span class="pre">shell</span> <span class="pre">--name</span> <span class="pre">&lt;daemon-name&gt;</span></code> like so:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">root</span><span class="nd">@myhostname</span> <span class="c1"># cephadm unit --name mon.myhostname stop</span>
<span class="n">root</span><span class="nd">@myhostname</span> <span class="c1"># cephadm shell --name mon.myhostname</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@myhostname</span> <span class="o">/</span><span class="p">]</span><span class="c1"># ceph-monstore-tool /var/lib/ceph/mon/ceph-myhostname get monmap &gt; monmap</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@myhostname</span> <span class="o">/</span><span class="p">]</span><span class="c1"># monmaptool --print monmap</span>
<span class="n">monmaptool</span><span class="p">:</span> <span class="n">monmap</span> <span class="n">file</span> <span class="n">monmap</span>
<span class="n">epoch</span> <span class="mi">1</span>
<span class="n">fsid</span> <span class="mi">28596</span><span class="n">f44</span><span class="o">-</span><span class="mi">3</span><span class="n">b56</span><span class="o">-</span><span class="mi">11</span><span class="n">ec</span><span class="o">-</span><span class="mi">9034</span><span class="o">-</span><span class="mi">482</span><span class="n">ae35a5fbb</span>
<span class="n">last_changed</span> <span class="mi">2021</span><span class="o">-</span><span class="mi">11</span><span class="o">-</span><span class="mi">01</span><span class="n">T20</span><span class="p">:</span><span class="mi">57</span><span class="p">:</span><span class="mf">19.755111</span><span class="o">+</span><span class="mi">0000</span>
<span class="n">created</span> <span class="mi">2021</span><span class="o">-</span><span class="mi">11</span><span class="o">-</span><span class="mi">01</span><span class="n">T20</span><span class="p">:</span><span class="mi">57</span><span class="p">:</span><span class="mf">19.755111</span><span class="o">+</span><span class="mi">0000</span>
<span class="n">min_mon_release</span> <span class="mi">17</span> <span class="p">(</span><span class="n">quincy</span><span class="p">)</span>
<span class="n">election_strategy</span><span class="p">:</span> <span class="mi">1</span>
<span class="mi">0</span><span class="p">:</span> <span class="p">[</span><span class="n">v2</span><span class="p">:</span><span class="mf">127.0.0.1</span><span class="p">:</span><span class="mi">3300</span><span class="o">/</span><span class="mi">0</span><span class="p">,</span><span class="n">v1</span><span class="p">:</span><span class="mf">127.0.0.1</span><span class="p">:</span><span class="mi">6789</span><span class="o">/</span><span class="mi">0</span><span class="p">]</span> <span class="n">mon</span><span class="o">.</span><span class="n">myhostname</span>
</pre></div>
</div>
<p>This command sets up the environment in a way that is suitable
for extended daemon maintenance and running the deamon interactively.</p>
</div>
<div class="section" id="restoring-the-mon-quorum">
<span id="cephadm-restore-quorum"></span><h2>Restoring the MON quorum<a class="headerlink" href="#restoring-the-mon-quorum" title="Permalink to this headline">¶</a></h2>
<p>In case the Ceph MONs cannot form a quorum, cephadm is not able
to manage the cluster, until the quorum is restored.</p>
<p>In order to restore the MON quorum, remove unhealthy MONs
form the monmap by following these steps:</p>
<ol class="arabic">
<li><p>Stop all MONs. For each MON host:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>ssh {mon-host}
cephadm unit --name mon.`hostname` stop
</pre></div>
</div>
</li>
<li><p>Identify a surviving monitor and log in to that host:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span>ssh {mon-host}
cephadm enter --name mon.`hostname`
</pre></div>
</div>
</li>
<li><p>Follow the steps in <a class="reference internal" href="../../rados/operations/add-or-rm-mons/#rados-mon-remove-from-unhealthy"><span class="std std-ref">从不健康集群删除监视器</span></a></p></li>
</ol>
</div>
<div class="section" id="manually-deploying-a-mgr-daemon">
<span id="cephadm-manually-deploy-mgr"></span><h2>Manually deploying a MGR daemon<a class="headerlink" href="#manually-deploying-a-mgr-daemon" title="Permalink to this headline">¶</a></h2>
<p>cephadm requires a MGR daemon in order to manage the cluster. In case the cluster
the last MGR of a cluster was removed, follow these steps in order to deploy
a MGR <code class="docutils literal notranslate"><span class="pre">mgr.hostname.smfvfd</span></code> on a random host of your cluster manually.</p>
<p>Disable the cephadm scheduler, in order to prevent cephadm from removing the new
MGR. See <a class="reference internal" href="../install/#cephadm-enable-cli"><span class="std std-ref">Different deployment scenarios</span></a>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ceph</span> <span class="n">config</span><span class="o">-</span><span class="n">key</span> <span class="nb">set</span> <span class="n">mgr</span><span class="o">/</span><span class="n">cephadm</span><span class="o">/</span><span class="n">pause</span> <span class="n">true</span>
</pre></div>
</div>
<p>Then get or create the auth entry for the new MGR:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ceph</span> <span class="n">auth</span> <span class="n">get</span><span class="o">-</span><span class="ow">or</span><span class="o">-</span><span class="n">create</span> <span class="n">mgr</span><span class="o">.</span><span class="n">hostname</span><span class="o">.</span><span class="n">smfvfd</span> <span class="n">mon</span> <span class="s2">&quot;profile mgr&quot;</span> <span class="n">osd</span> <span class="s2">&quot;allow *&quot;</span> <span class="n">mds</span> <span class="s2">&quot;allow *&quot;</span>
</pre></div>
</div>
<p>Get the ceph.conf:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ceph</span> <span class="n">config</span> <span class="n">generate</span><span class="o">-</span><span class="n">minimal</span><span class="o">-</span><span class="n">conf</span>
</pre></div>
</div>
<p>Get the container image:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">ceph</span> <span class="n">config</span> <span class="n">get</span> <span class="s2">&quot;mgr.hostname.smfvfd&quot;</span> <span class="n">container_image</span>
</pre></div>
</div>
<p>Create a file <code class="docutils literal notranslate"><span class="pre">config-json.json</span></code> which contains the information neccessary to deploy
the daemon:</p>
<div class="highlight-json notranslate"><div class="highlight"><pre><span></span><span class="p">{</span>
  <span class="nt">&quot;config&quot;</span><span class="p">:</span> <span class="s2">&quot;# minimal ceph.conf for 8255263a-a97e-4934-822c-00bfe029b28f\n[global]\n\tfsid = 8255263a-a97e-4934-822c-00bfe029b28f\n\tmon_host = [v2:192.168.0.1:40483/0,v1:192.168.0.1:40484/0]\n&quot;</span><span class="p">,</span>
  <span class="nt">&quot;keyring&quot;</span><span class="p">:</span> <span class="s2">&quot;[mgr.hostname.smfvfd]\n\tkey = V2VyIGRhcyBsaWVzdCBpc3QgZG9vZi4=\n&quot;</span>
<span class="p">}</span>
</pre></div>
</div>
<p>Deploy the daemon:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cephadm</span> <span class="o">--</span><span class="n">image</span> <span class="o">&lt;</span><span class="n">container</span><span class="o">-</span><span class="n">image</span><span class="o">&gt;</span> <span class="n">deploy</span> <span class="o">--</span><span class="n">fsid</span> <span class="o">&lt;</span><span class="n">fsid</span><span class="o">&gt;</span> <span class="o">--</span><span class="n">name</span> <span class="n">mgr</span><span class="o">.</span><span class="n">hostname</span><span class="o">.</span><span class="n">smfvfd</span> <span class="o">--</span><span class="n">config</span><span class="o">-</span><span class="n">json</span> <span class="n">config</span><span class="o">-</span><span class="n">json</span><span class="o">.</span><span class="n">json</span>
</pre></div>
</div>
</div>
<div class="section" id="analyzing-core-dumps">
<h2>Analyzing core dumps<a class="headerlink" href="#analyzing-core-dumps" title="Permalink to this headline">¶</a></h2>
<p>In case a Ceph daemon crashes, cephadm supports analyzing core dumps. To enable core dumps, run</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span class="prompt1"><span class="nb">ulimit</span> -c unlimited</span>
</pre></div></div><p>core dumps will now be written to <code class="docutils literal notranslate"><span class="pre">/var/lib/systemd/coredump</span></code>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>core dumps are not namespaced by the kernel, which means
they will be written to <code class="docutils literal notranslate"><span class="pre">/var/lib/systemd/coredump</span></code> on
the container host.</p>
</div>
<p>Now, wait for the crash to happen again. (To simulate the crash of a daemon, run e.g. <code class="docutils literal notranslate"><span class="pre">killall</span> <span class="pre">-3</span> <span class="pre">ceph-mon</span></code>)</p>
<p>Install debug packages by entering the cephadm shell and install <code class="docutils literal notranslate"><span class="pre">ceph-debuginfo</span></code>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="c1"># cephadm shell --mount /var/lib/systemd/coredump</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@host1</span> <span class="o">/</span><span class="p">]</span><span class="c1"># dnf install ceph-debuginfo gdb zstd</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@host1</span> <span class="o">/</span><span class="p">]</span><span class="c1"># unzstd /mnt/coredump/core.ceph-*.zst</span>
<span class="p">[</span><span class="n">ceph</span><span class="p">:</span> <span class="n">root</span><span class="nd">@host1</span> <span class="o">/</span><span class="p">]</span><span class="c1"># gdb /usr/bin/ceph-mon /mnt/coredump/core.ceph-...</span>
<span class="p">(</span><span class="n">gdb</span><span class="p">)</span> <span class="n">bt</span>
<span class="c1">#0  0x00007fa9117383fc in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0</span>
<span class="c1">#1  0x00007fa910d7f8f0 in std::condition_variable::wait(std::unique_lock&lt;std::mutex&gt;&amp;) () from /lib64/libstdc++.so.6</span>
<span class="c1">#2  0x00007fa913d3f48f in AsyncMessenger::wait() () from /usr/lib64/ceph/libceph-common.so.2</span>
<span class="c1">#3  0x0000563085ca3d7e in main ()</span>
</pre></div>
</div>
</div>
</div>



           </div>
           
          </div>
          <footer>
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
        <a href="../../dev/cephadm/" class="btn btn-neutral float-right" title="CEPHADM 开发者文档" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right" aria-hidden="true"></span></a>
        <a href="../client-setup/" class="btn btn-neutral float-left" title="Basic Ceph Client Setup" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left" aria-hidden="true"></span> Previous</a>
    </div>

  <hr/>

  <div role="contentinfo">
    <p>
        &#169; Copyright 2016, Ceph authors and contributors. Licensed under Creative Commons Attribution Share Alike 3.0 (CC-BY-SA-3.0).

    </p>
  </div> 

</footer>
        </div>
      </div>

    </section>

  </div>
  

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
   

</body>
</html>